package com.coderfans.datacsdn.parse;

import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;

import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.concurrent.LinkedBlockingDeque;

/**
 * Created by ${fanchunshuai} on 2017-8-11.
 *
 * @version: V1.0
 * @Desc:
 * @Copyright (c) 2017 58到家-版权所有
 */
public class MultParseUserExt {
    public static void main(String[] args) throws InterruptedException {
        long start = System.currentTimeMillis();
        LinkedBlockingDeque<String> list = new LinkedBlockingDeque<String>(700000);

        File file = new File("E:\\spiderFile\\csdnUserLink.txt");
        try {
            List<String> useList = FileUtils.readLines(file,"utf-8");
            for (String urls : useList){
                String [] urlArray = urls.split(",");
                for (String url : urlArray){
                    if(StringUtils.isBlank(url)){
                        continue;
                    }
                    list.put(url);
                }
            }
        } catch (IOException e) {
            e.printStackTrace();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }

        System.out.println(list.size());
        for (int i =0 ;i< 8;i++){
            ParseUserExtTask task = new ParseUserExtTask(list);
            Thread thread= new Thread(task);
            thread.start();
            thread.join();
        }

        long end = System.currentTimeMillis();
        System.out.println((end - start) /1000+"s");

    }
}
